# coding: utf8

import os
import logging
import time
import re
import traceback
import requests_html
from multiprocessing.pool import ThreadPool


logging.basicConfig(level=logging.DEBUG)

session = requests_html.HTMLSession()
pool = ThreadPool(os.cpu_count() * 2)  # 要使用 hanlp, 线程数设为1
host = "www.sic.tsinghua.edu.cn"
reduce_breakline = re.compile(r"\s{2,}")

# import hanlp
# HanLP = hanlp.load(hanlp.pretrained.mtl.CLOSE_TOK_POS_NER_SRL_DEP_SDP_CON_ELECTRA_BASE_ZH)
# print(HanLP.tasks.keys())
# doc = HanLP(content, tasks=['ner'])


addr_ptn = re.compile(r'(?:通信地址|办公地址|Address)(?::|：)([\s\S]*?\n)')
tel_ptn = re.compile(r'(?:办公电话|电话|联系方式|Tel|tek)(?::|：)([0-9+\-]*?\n)')
email_ptn = re.compile(r'(?:电子邮箱|E\-mail|Email|email|e-mail|邮箱地址)(?::|：)(\S+@\S+\.\S+?\n)')
ri_ptn = re.compile(r'研究(?:方向|领域|重点)(?:为|包括)*(?::|：)*([\s\S]*?[。|\n])')
pos_ptn = re.compile(r"(教授|副教授|助理教授|研究员|副研究员|博士|(?:硕士|博士|研究生|博士生)导师|IEEE Fellow)")
edu_ptn = re.compile(r"(?:考入|攻读|进修|毕业于|学位|博士后)")
split_ptn = re.compile(r'([,，\.。\n])')


def teacher_list():
    try:
        r = session.get("http://www.sic.tsinghua.edu.cn/szdw/rygc/szxx.htm")
        if r.status_code == 200:
            li = r.html.xpath('//div[contains(@class, "list_zhy")][1]/a')

            teachers = []
            for a in li:
                name = a.text

                if a.absolute_links:
                    url = list(a.absolute_links)[0]
                    teachers.append((name, url))
                else:
                    logging.warning("%s url is none.", name)

            pool.imap_unordered(teacher_detail, teachers)

        else:
            logging.info("wrong status code: %s", r.status_code)
            r.raise_for_status()
    except Exception as e:
        traceback.print_exc()
        raise e
    pass


def teacher_detail(payload):
    name, url = payload
    logging.info("extract teacher %s ...", name)

    try:
        r = session.get(url)
        if r.status_code == 200:
            infos = r.html.xpath("//div[@id='vsb_content_2']")[0]
            content = reduce_breakline.sub("\n", infos.full_text)
            # print(content)

            avatar = r.html.xpath("//*[contains(@class, 'v_news_content')][1]//img[1]/@src")
            avatar = host + avatar[0] if avatar else ''

            addr, tel, email, ri = [ptn.search(content) for ptn in (addr_ptn, tel_ptn, email_ptn, ri_ptn)]
            addr, tel, email, ri = [(res.group(1).strip() if res else '') for res in (addr, tel, email, ri)]

            basic_info = []
            intros = []
            for p in infos.xpath("//p"):
                if len(p.full_text) > 32:
                    intros.append(p.full_text)
                else:
                    basic_info.append(p.full_text)
            intros = "\n".join(intros)
            basic_info = "\n".join(basic_info)

            # find position
            pos = pos_ptn.findall(basic_info)
            if not pos:
                ptn = r"(%s[\s\S]*?。)" % name
                first_sent = re.search(ptn, intros)
                if first_sent:
                    first_sent = first_sent.group(1)
                    pos = pos_ptn.findall(first_sent)
            pos = ", ".join(list(set(pos)))

            # find education
            edu_sents = []
            sentences = split_ptn.split(intros)
            for sent in sentences:
                res = edu_ptn.search(sent)
                if res:
                    edu_sents.append(sent)
            edu = "，".join(edu_sents) + "。" if edu_sents else ""

            org = "清华大学"
            department = "集成电路学院"

            logging.info("name: %s, pos: %s, addr: %s, tel: %s, email: %s, org: %s, dep: %s, avatar: %s, ri: %s, edu: %s", name, pos, addr, tel, email, org, department, avatar, ri, edu)
            pass
        else:
            logging.info("wrong status code: %s, %s, %s", r.status_code, name, url)
            r.raise_for_status()
    except Exception as e:
        traceback.print_exc()
        raise e


if __name__ == '__main__':
    teacher_list()

    pool.close()
    pool.join()
    session.close()
    pass




